package com.chance.cc.crawler.development.scripts.taobao;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.meta.core.bean.crawler.CrawlerDomainUrls;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.Option;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_Domain_Result_Json;
import static com.chance.cc.crawler.development.scripts.allfeild.AICCommonField.Tag_Field_ECommerce_List_Content;

public class TaobaoProductDetailCrawlerScript extends CrawlerCommonScript {

    private Logger log = LoggerFactory.getLogger(TaobaoProductDetailCrawlerScript.class);

    public static final String site = "product";
    public static final String domain = "taobao";

    private Downloader httpDownload;
    private static Object httpDownloadObj = new Object();
    public static String startUrl = "https://trade-acs.m.taobao.com/gw/mtop.taobao.detail.getdetail/6.0/";
    public static final String tbProduceUrlRegular = "https://\\S*/gw/mtop.taobao.detail.getdetail/6.0/\\S*";
    public static final String tbProduceUrl = "https://trade-acs.m.taobao.com/gw/mtop.taobao.detail.getdetail/6.0/?data=";
    public static final String tbProduceUrlParamFormat = "{\"detail_v\":\"3.3.2\",\"exParams\":\"{\\\"\\\":\\\"\\\",\\\"ad_type\\\":\\\"1.0\\\"," +
            "\\\"appReqFrom\\\":\\\"detail\\\",\\\"clientCachedTemplateKeys\\\":\\\"[{\\\\\\\"id\\\\\\\":\\\\\\\"1538383035450\\\\\\\"" +
            ",\\\\\\\"version\\\\\\\":\\\\\\\"138\\\\\\\"}]\\\",\\\"container_type\\\":\\\"xdetail\\\",\\\"countryCode\\\":\\\"CN\\\"" +
            ",\\\"cpuCore\\\":\\\"4\\\",\\\"cpuMaxHz\\\":\\\"1996800\\\",\\\"detailAlgoParam\\\":\\\"%%E5%%A5%%B3%%E5%%86%%AC%%E5%%A4%%96%%E5%%A5%%97\\\"" +
            ",\\\"dinamic_v3\\\":\\\"true\\\",\\\"from\\\":\\\"search\\\",\\\"id\\\":\\\"%s\\\",\\\"itemId\\\":\\\"%s\\\"," +
            "\\\"item_id\\\":\\\"%s\\\",\\\"latitude\\\":\\\"31.252115\\\",\\\"list_type\\\":\\\"search\\\",\\\"locType\\\":\\\"\\\"," +
            "\\\"longitude\\\":\\\"121.589279\\\",\\\"nick\\\":\\\"tb288552944\\\",\\\"osVersion\\\":\\\"26\\\",\\\"phoneType\\\":\\\"MI+5s\\\"," +
            "\\\"skuId\\\":\\\"\\\",\\\"soVersion\\\":\\\"2.0\\\",\\\"spm\\\":\\\"a2141.7631557.itemlist.11\\\",\\\"spm-cnt\\\":\\\"a2141.7631564\\\"," +
            "\\\"store_id\\\":\\\"\\\",\\\"supportV7\\\":\\\"true\\\",\\\"ultron2\\\":\\\"true\\\",\\\"utdid\\\":\\\"%s\\\"}\"," +
            "\"itemNumId\":\"%s\"}";

    public static final String deviceId = "AlChtDgjeNqK62mIapWOaPAkN49ZwTldvJQ-3bbjeEB2";
//    public static final String sid = "224ab95d09f4ca5a5499fed4c96f2024";
    public static final String ttid = "1568860058617@taobao_android_10.0.0";
//    public static final String uid = "2211481958764";
    public static final String utdid = "YLWtjXHbDnsDAPQgIVKHUarl";
    public static String globalHttpSite = "product_1";
    public static Object siteObject = new Object();

    /**
     * 脚本domain定义
     * @return
     */
    public String domain() {
        return domain;
    }

    /**
     * 进入脚本的正则列表
     */
    public void initUrlRegulars() {
        addUrlRegular(tbProduceUrlRegular);
    }

    /**
     * 是否执行脚本 ： 输入数据检查，合格的才进入脚本
     * @param crawlerRequestRecord
     * @return
     */
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRequestRecord) {
        return true;
    }

    @Override
    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord requestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> allItemRecords = new ArrayList<>();

        HttpRequest httpRequest = requestRecord.getHttpRequest();
        try {
            if(requestRecord.tagsCreator().bizTags().hasKVTag(Tag_Field_Domain_Result_Json)){
                requestRecord.setNeedWashPage(true);
                KVTag domainResultJson = requestRecord.tagsCreator().bizTags().getCategoryTag().getKVTag(Tag_Field_Domain_Result_Json);
                CrawlerDomainUrls crawlerDomainUrls = JSON.parseObject(String.valueOf(domainResultJson.getVal()),CrawlerDomainUrls.class);
                String url = crawlerDomainUrls.getUrl();
                Json urlJson = new Json(url);
                String itemId= urlJson.jsonPath($_type + ".itemId").get();
                String tbProduceUrlParam = String.format(tbProduceUrlParamFormat, itemId,itemId,itemId,utdid,itemId);
                String encodeParam = URLEncoder.encode(tbProduceUrlParam,"utf-8").replaceAll("%2B","+");
                httpRequest.setUrl(tbProduceUrl + encodeParam);
                fillHttpRequestHeaders(httpDownload,httpRequest,encodeParam);
                requestRecord.getHttpConfig().setHttpSite(globalHttpSite);
                requestRecord.setDownload(true);
                requestRecord.setNeedParsedPage(true);
                requestRecord.setNeedWashPage(true);
                requestRecord.setSkipPipeline(false);
                requestRecord.tagsCreator().bizTags().addCustomKV(Tag_Field_ECommerce_List_Content,url);
//            requestRecord.tagsCreator().bizTags().getCategoryTag().getKvTags().remove(Tag_Field_Domain_Result_Json); //移除
            }
        } catch (Exception e) {
            log.error(e.getMessage());
        }
        return allItemRecords;
    }

    private void fillHttpRequestHeaders(Downloader httpDownload, HttpRequest httpRequest,String encodeParam) throws UnsupportedEncodingException {
        long t_long = System.currentTimeMillis()/1000;
        String t = "" + t_long;
        Map<String, String> signatureMap = tbSignature(httpDownload, URLDecoder.decode(encodeParam, "UTF-8"), deviceId, t,
                ttid, utdid,"r_142");
//        httpRequest.addHeader("x-sid",URLEncoder.encode(sid,"utf-8"));
//        httpRequest.addHeader("x-uid",URLEncoder.encode(uid,"utf-8"));
        httpRequest.addHeader("x-pv","6.3");
        httpRequest.addHeader("x-features","27");
        httpRequest.addHeader("content-type","application/x-www-form-urlencoded;charset=UTF-8");
        httpRequest.addHeader("x-t",t);
        httpRequest.addHeader("x-extdata","openappkey%3DDEFAULT_AUTH");
        httpRequest.addHeader("x-ttid",URLEncoder.encode(ttid,"utf-8"));
//        httpRequest.addHeader("x-location","121.589279%2C31.252115");
        httpRequest.addHeader("x-utdid",URLEncoder.encode(utdid,"utf-8"));
        httpRequest.addHeader("x-appkey","21646297");
        httpRequest.addHeader("x-devid",URLEncoder.encode(deviceId,"utf-8"));
        httpRequest.addHeader("Host","trade-acs.m.taobao.com");
        httpRequest.getHeaders().putAll(signatureMap);
    }

    private Map<String,String> tbSignature(Downloader downloader, String data, String deviceId, String t,
                                                  String ttid, String utdid, String str3){

        Map<String,String> dataMap = new HashMap<>();
        dataMap.put("api","mtop.taobao.detail.getdetail");
        dataMap.put("appKey","21646297");
        dataMap.put("data",data);
        dataMap.put("deviceId",deviceId);
        dataMap.put("extdata","openappkey=DEFAULT_AUTH");
//        dataMap.put("lat","31.252115");
//        dataMap.put("lng","121.589279");
//        dataMap.put("sid",sid);
        dataMap.put("t",t);
        dataMap.put("ttid",ttid);
//        dataMap.put("uid",uid);
        dataMap.put("utdid",utdid);
        dataMap.put("v","6.0");
        dataMap.put("x-features","27");

        Map<String,String> pageMap = new HashMap<>();
        pageMap.put("pageId","http://item.taobao.com/item.htm");
        pageMap.put("pageName","com.taobao.android.detail.wrapper.activity.DetailActivity");

        String signatureUrl = "http://192.168.1.210:8384/frida/tb/signature";
        HttpConfig tbSignatureConfig = HttpConfig.me("signature");
        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl(signatureUrl);
        Map<String,Object> tbDataMap = new HashMap<>();
        tbDataMap.put("map1",JSON.toJSONString(dataMap));
        tbDataMap.put("map2",JSON.toJSONString(pageMap));
        tbDataMap.put("str1",21646297);
        tbDataMap.put("str2","null");
        tbDataMap.put("b1",false);
        tbDataMap.put("str3",str3);
        httpRequest.setRequestBody(HttpRequestBody.form(tbDataMap,"utf-8"));
        httpRequest.setMethod("POST");
        HttpPage download = downloader.download(httpRequest, tbSignatureConfig);
        String rawText = download.getRawText();
        Map signatureMap = JSON.parseObject(rawText, Map.class);
        Map<String,String> encodeSignatureMap = new HashMap<>();
        for (Object objKey : signatureMap.keySet()) {
            String key = objKey.toString();
            String value = signatureMap.get(key).toString();
            try {
                value = URLEncoder.encode(value,"utf-8");
                encodeSignatureMap.put(key,value);
            } catch (UnsupportedEncodingException e) {
                log.error(e.getMessage());
            }
        }

        return encodeSignatureMap;
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> crawlerRequestRecords = new ArrayList<>();

        if (!page.isDownloadSuccess()){
            log.error("TB product detail proxy has error ,will retry");
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
            return crawlerRequestRecords;
        }

        try {
            Json json = new Json(page.getRawText());
            Configuration conf = Configuration.defaultConfiguration().addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL,Option.SUPPRESS_EXCEPTIONS);
            json.setConfiguration(conf);
            String value = json.jsonPath($_type + ".data.apiStack[0].value").get();
            if (value == null){
                String trade = json.jsonPath($_type + ".data.trade").get();
                if (StringUtils.isNotBlank(trade) && trade.contains("/expired")){
                    String itemInfo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(Tag_Field_Domain_Result_Json);
                    log.warn("tb product {} expired!",itemInfo);
                    crawlerRecord.setNeedWashPage(false); //不需要清洗页面
                } else {
//                    String ret = json.jsonPath($_type + ".ret").get();
//                    if (StringUtils.isNotBlank(ret) && (ret.contains("SESSION失效") || ret.contains("请检查是否使用了代理软件或VPN"))){
//                            log.error("tb product detail,http site {}, ret {}",crawlerRecord.getHttpConfig().getHttpSite(),ret);
//                        changeHttpSite(crawlerRecord.getHttpConfig());
//                        addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
//                        crawlerRecord.setNeedWashPage(false); //不需要清洗页面
//                    } else {
//                        throw new Exception("tb product detail download error");
//                    }
                    throw new Exception("tb product detail download error");
                }
            }
        } catch (Exception e) {
            log.error("taobao product detail download failed,error message {},request detail {},page rawText {}"
                    ,e.getMessage(),JSON.toJSONString(page.getRequest()),page.getRawText());
            addCrawlerRecords(crawlerRequestRecords,crawlerRecord);
            crawlerRecord.setNeedWashPage(false); //下载异常，不需要清洗页面
        }

        return crawlerRequestRecords;
    }

    private void changeHttpSite(HttpConfig httpConfig){
        String httpSite = httpConfig.getHttpSite();
        if (globalHttpSite.equals(httpSite)){
            synchronized (siteObject) {
                if (globalHttpSite.equals(httpSite)){
                    String siteSuffix = RandomStringUtils.random(8, "abcdefghijklmnopqrstuvwxyz1234567890");
                    globalHttpSite = "product_"+siteSuffix;
                    log.warn("tb product detail http env has exception change http site:" + globalHttpSite);
                }
            }
        }
    }

    @Override
    public void beforeDownload(CrawlerRecordContext context) {
        if (httpDownload == null){
            synchronized (httpDownloadObj) {
                if (httpDownload == null){
                    httpDownload = context.getPageDownloader();
                }
            }
        }
        super.beforeDownload(context);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {

        List<CrawlerData> crawlerDataList = new ArrayList<>();
        try {
            Json json = new Json(page.getRawText());
            String itemId = json.jsonPath($_type + ".data.item.itemId").get();

            CrawlerData crawlerArticleData = CrawlerData.builder()
                    .data(crawlerRecord,page)
                    .dataId(StringUtils.joinWith("-",domain,site,itemId))
                    .releaseTime(System.currentTimeMillis())
                    .content(page.getRawText())
                    .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                    .build();

            crawlerDataList.add(crawlerArticleData);

        } catch (Exception e) {
            log.error(e.getMessage(),e);
        }
        return crawlerDataList;
    }

    public void afterExecute(CrawlerRecordContext crawlerRecordContext) {

    }

    private void addCrawlerRecords(List<CrawlerRequestRecord> crawlerRequestRecords,CrawlerRequestRecord crawlerRecord){

        int count = 1;
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        if (crawlerBusinessTags.hasKVTag("taobao_download_retry_count")){
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag("taobao_download_retry_count").getVal();
            if (count >= 20){
                log.error("taobao product detail download he number of retries exceeds the limit" +
                        ",request url {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        count++;
        crawlerBusinessTags.addCustomKV("taobao_download_retry_count",count);

        CrawlerRequestRecord crawlerRequestRecord = CrawlerRequestRecord.builder()
                .turnPageRequest(crawlerRecord)
                .httpRequest(crawlerRecord.getHttpRequest())
                .needWashed(true)
                .releaseTime(System.currentTimeMillis())
                .copyBizTags()
                .notFilterRecord()
                .build();
        crawlerRequestRecords.add(crawlerRequestRecord);
    }
}
